##############
# (*) Kaleidoscope
# By: Allan Spale
# May 2008

from bsddb import db
import sys
import mx.DateTime

from Kaleidoscope.KSColumnBSDDBUnique import KSColumnBSDDBUnique
from Kaleidoscope.KSColumn import KSColumn, KSColumnError
from Kaleidoscope.KSEventManager import KSEventManager
from Kaleidoscope.KSUtility import KSUtility
from Kaleidoscope.KSCursorError import KSCursorError


class KSColumnBSDDBDuplicate(KSColumnBSDDBUnique):
    """
    This is a class that be subclassed by data stores that are not related
    to BerkeleyDB. Potential systems for which to include support include
    Metaweb's Freebase and Google Spreadsheets.

    Because of complexities with associated secondary databases and because
    these secondary databases can themselves be primary databases with
    some other secondary databases attached to it...no associated secondary
    databases will be used...just to make things a bit simpler. There will be
    a KSMappedColumn class that will associate the data of the
    KSDuplicateColumn withits type and the related database containing the
    items that are part of that type.

    Since BTree keys tend to sort better with a big endian encoding, keys
    (if they are numerical) will be stored as big endian. Other data objects
    will be stored in their local endian format. When transmitted remotely,
    data will be encoded using XML-RPC style with base64 used for arbitrary
    binary data.

    Note, some of this code is redundant, but the redundancy occurs to try
    to cut down on some unnecessary comparisons...which could add up over time.

    To allow this class to act as an inverse index, another parameter was
    added to the initialization constructor that would indicate whether or
    not keys should be packed. This flag will allow all data to be pickled
    since keys in an inverse index may not be suitable for packing (due to
    the fact that packing involves knowing the precise data structure types
    in advance).

    """
    # Class constants
    __KEY = 0
    __VALUE = 1

    # Event IDs ... redundant, but code will be safer if constants are
    # assigned and reused throughout the code
    __KSEM_BEF_set_flags = 120
    __KSEM_BEF_get_data_from_key = 121
    __KSEM_BEF_get_keys_from_data_value = 122
    __KSEM_BEF_get_keys_from_data_object_range = 123
    __KSEM_BEF_get_data_objects_from_key_range = 124
    __KSEM_BEF_has_value = 125
    __KSEM_BEF_insert_item_with_key_value_pair = 126
    __KSEM_BEF_insert_value_and_generate_key = 127
    __KSEM_BEF_insert_value_list_and_generate_key = 128
    __KSEM_BEF_change_key = 129
    __KSEM_BEF_change_key_value_pair = 130
    __KSEM_BEF_delete_items_by_value_list = 131
    __KSEM_BEF_print_all = 132
    __KSEM_BEF_insert_items_with_key_and_value_list = 133
    __KSEM_BEF_reorganize_keys = 134

    __KSEM_AFT_set_flags = 1120
    __KSEM_AFT_get_data_from_key = 1121
    __KSEM_AFT_get_keys_from_data_value = 1122
    __KSEM_AFT_get_keys_from_data_object_range = 1123
    __KSEM_AFT_get_data_objects_from_key_range = 1124
    __KSEM_AFT_has_value = 1125
    __KSEM_AFT_insert_item_with_key_value_pair = 1126
    __KSEM_AFT_insert_value_and_generate_key = 1127
    __KSEM_AFT_insert_value_list_and_generate_key = 1128
    __KSEM_AFT_change_key = 1129
    __KSEM_AFT_change_key_value_pair = 1130
    __KSEM_AFT_delete_items_by_value_list = 1131
    __KSEM_AFT_print_all = 1132
    __KSEM_AFT_insert_items_with_key_and_value_list = 1133
    __KSEM_AFT_reorganize_keys = 1134


    def __init__(self, u_column_name, i_column_flags, mx_start_datetime,
                  bsddb_environment, u_key_type, b_pack_key):

        try:
            super(KSColumnBSDDBDuplicate, self).__init__(u_column_name,
                i_column_flags, mx_start_datetime, bsddb_environment,
                u_key_type, b_pack_key, KSColumn.B_DUPLICATE_VALUES)

            #self.__id = uuid.uuid4()
            #self.ksem_event_manager = KSEventManager(self.__id)
            self.ksem_event_manager.create_function_event(
                u'set_flags',
                KSColumnBSDDBDuplicate.__KSEM_BEF_set_flags,
                KSColumnBSDDBDuplicate.__KSEM_AFT_set_flags)
            self.ksem_event_manager.create_function_event(
                u'get_data_from_key',
                KSColumnBSDDBDuplicate.__KSEM_BEF_get_data_from_key,
                KSColumnBSDDBDuplicate.__KSEM_AFT_get_data_from_key)
            self.ksem_event_manager.create_function_event(
                u'get_keys_from_data_value',
                KSColumnBSDDBDuplicate.__KSEM_BEF_get_keys_from_data_value,
                KSColumnBSDDBDuplicate.__KSEM_AFT_get_keys_from_data_value)
            self.ksem_event_manager.create_function_event(
                u'get_keys_from_data_object_range',
                KSColumnBSDDBDuplicate.__KSEM_BEF_get_keys_from_data_object_range,
                KSColumnBSDDBDuplicate.__KSEM_AFT_get_keys_from_data_object_range)
            self.ksem_event_manager.create_function_event(
                u'get_data_objects_from_key_range',
                KSColumnBSDDBDuplicate.__KSEM_BEF_get_data_objects_from_key_range,
                KSColumnBSDDBDuplicate.__KSEM_AFT_get_data_objects_from_key_range)
            self.ksem_event_manager.create_function_event(
                u'has_value', KSColumnBSDDBDuplicate.__KSEM_BEF_has_value,
                KSColumnBSDDBDuplicate.__KSEM_AFT_has_value)
            self.ksem_event_manager.create_function_event(
                u'insert_item_with_key_value_pair',
                KSColumnBSDDBDuplicate.__KSEM_BEF_insert_item_with_key_value_pair,
                KSColumnBSDDBDuplicate.__KSEM_AFT_insert_item_with_key_value_pair)
            self.ksem_event_manager.create_function_event(
                u'insert_value_and_generate_key',
                KSColumnBSDDBDuplicate.__KSEM_BEF_insert_value_and_generate_key,
                KSColumnBSDDBDuplicate.__KSEM_AFT_insert_value_and_generate_key)
            self.ksem_event_manager.create_function_event(
                u'insert_value_list_and_generate_key',
                KSColumnBSDDBDuplicate.__KSEM_BEF_insert_value_list_and_generate_key,
                KSColumnBSDDBDuplicate.__KSEM_AFT_insert_value_list_and_generate_key)
            self.ksem_event_manager.create_function_event(
                u'change_key', KSColumnBSDDBDuplicate.__KSEM_BEF_change_key,
                KSColumnBSDDBDuplicate.__KSEM_AFT_change_key)
            self.ksem_event_manager.create_function_event(
                u'change_key_value_pair',
                KSColumnBSDDBDuplicate.__KSEM_BEF_change_key_value_pair,
                KSColumnBSDDBDuplicate.__KSEM_AFT_change_key_value_pair)
            self.ksem_event_manager.create_function_event(
                u'delete_items_by_value_list',
                KSColumnBSDDBDuplicate.__KSEM_BEF_delete_items_by_value_list,
                KSColumnBSDDBDuplicate.__KSEM_AFT_delete_items_by_value_list)
            self.ksem_event_manager.create_function_event(
                u'print_all', KSColumnBSDDBDuplicate.__KSEM_BEF_print_all,
                KSColumnBSDDBDuplicate.__KSEM_AFT_print_all)
            self.ksem_event_manager.create_function_event(
                u'insert_items_with_key_and_value_list',
                KSColumnBSDDBDuplicate.__KSEM_BEF_insert_items_with_key_and_value_list,
                KSColumnBSDDBDuplicate.__KSEM_AFT_insert_items_with_key_and_value_list)

            if (isinstance(i_column_flags, int)):
                i_column_flags = i_column_flags | db.DB_DUPSORT
                self.set_flags(i_column_flags)
            else:
                KSColumnError.print_and_raise_error(
                    u'Flags for the database must be stored in an integer.',
                    KSColumnError.B_CLASS_EXCEPTION)

        except db.DBError:
            KSColumnError.print_and_raise_error( u'The new database could not'
                + u' be initialized.', KSColumnError.B_UNKNOWN_EXCEPTION)

        # Stuff for later...
        # will be important especially with the way the database columns
        # will be so connected
        """
        self.__ctIdentifier.registerSubclassToReceiveEvents(
            self.__processEventChannel )

        # Decide which events you want to respond to from CoreTable and ContainerTable
        self.__dictChannelEvents = {}

        """
    """
    def set_transaction_id(self, i_transaction_id):
        return super(KSColumnBSDDBDuplicate, self).set_transaction_id(
            i_transaction_id)
    """


    def set_flags(self, i_column_flags):
        i_before_event_id = KSColumnBSDDBDuplicate.__KSEM_BEF_set_flags
        i_after_event_id = KSColumnBSDDBDuplicate.__KSEM_AFT_set_flags
        self.ksem_event_manager.send_event_message(
            i_before_event_id, [i_column_flags])

        i_column_flags = i_column_flags | db.DB_DUPSORT | db.DB_DUP
        super(KSColumnBSDDBDuplicate, self).set_flags(i_column_flags)

        self.ksem_event_manager.send_event_message(
            i_after_event_id, [True])
        return True


    def get_data_from_key(self, obj_key, kstx_parent_transaction,
                              b_return_packed_data):
        """
        Return a list of data objects based on the value of the key.

        """
        try:
            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.get_data_from_key)
            i_before_event_id = KSColumnBSDDBDuplicate.__KSEM_BEF_get_data_from_key
            i_after_event_id = KSColumnBSDDBDuplicate.__KSEM_AFT_get_data_from_key
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [obj_key, b_return_packed_data])

            obj_list = []
            cursor = self.get_cursor(kstx_transaction)

            # Searching for a NULL_VALUE key will return a NULL_VALUE
            if (obj_key == KSColumn.NULL_VALUE):
                obj_list.append(KSColumn.NULL_VALUE)
                packed_key = obj_key
                cursor.get = lambda x, y, z: None
            else:
                packed_key = self.marshal_key(obj_key)

            record = cursor.get(packed_key, None, db.DB_SET)

            if (b_return_packed_data):
                while (record):
                    obj_list.append(record[self.__VALUE])
                    record = cursor.next_dup()
                # end loop
            else:
                while (record):
                    obj_unpickled_data = self.unpickle_data(
                        record[self.__VALUE])
                    obj_list.append(obj_unpickled_data)
                    record = cursor.next_dup()
                # end loop

            cursor.close()
            kstx_transaction.commit(0, self.get_data_from_key)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_key, obj_list])
            return obj_list

        except db.DBNotFoundError:
            # Just return an empty list
            cursor.close()
            kstx_transaction.cancel()
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_key, []])

        except KSCursorError:
            # No data will be found because the database is empty
            kstx_transaction.commit(0, self.get_data_from_key)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_key, obj_list])
            return obj_list


    def get_keys_from_data_value(self, obj_target_data, kstx_parent_transaction,
                                     b_return_packed_data):
        """
        Go through the data and find the list of keys from the given values.
        Since keys may contain duplicate data items, exhaustively search
        the entire database (including lists for each key).

        """
        try:
            # OK to look for NULL_VALUE objects

            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.get_keys_from_data_value)
            i_before_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_BEF_get_keys_from_data_value
            i_after_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_AFT_get_keys_from_data_value
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [obj_target_data, b_return_packed_data])

            #obj_pickled_target_data = self.pickle_data(obj_target_data)
            list_keys = []
            b_searching = True

            cursor = self.get_cursor(kstx_transaction)
            record = cursor.first()

            if (b_return_packed_data):
                optional_data_unpacking = \
                    lambda item_packed_key, u_key_type: item_packed_key
            else:
                optional_data_unpacking = lambda item_packed_key, u_key_type: \
                    self.unmarshal_key(item_packed_key, u_key_type)

            while (record):
                while (b_searching):
                    unpickled_record_data = \
                        self.unpickle_data(record[self.__VALUE])
                    if (obj_target_data == unpickled_record_data):
                        list_keys.append(optional_data_unpacking(
                            record[self.__KEY], self.u_key_type))
                        b_searching = False
                    else:
                        record = cursor.next_dup()
                        if (not record):
                            b_searching = False
                        # end if
                    # end if
                # end inner loop
                record = cursor.next()
                b_searching = True
            # end outer loop

            cursor.close()
            kstx_transaction.commit(0, self.get_keys_from_data_value)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_target_data, list_keys])
            return list_keys

        except db.DBNotFoundError:
            cursor.close()
            kstx_transaction.cancel()
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_target_data, []])

        except KSCursorError:
            # No keys are returned because the database is empty
            kstx_transaction.commit(0, self.get_keys_from_data_value)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_target_data, list_keys])
            return list_keys



    def get_keys_from_data_object_range(self, obj_start_data, obj_end_data,
            i_low_comparison_operator, i_high_comparison_operator,
            kstx_parent_transaction, b_return_packed_data):
        """
        Go through the data and find the list of keys from the given values
        in the range of data items. It is assumed that the caller will use
        a type that would support a range of data. Additionally, it is
        assumed that obj_data has been packed prior to calling this
        function. Range is obj_start_data <= data_item <= obj_end_data.

        """
        try:
            # OK to look for NULL_VALUE objects

            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.get_keys_from_data_object_range)
            i_before_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_BEF_get_keys_from_data_object_range
            i_after_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_AFT_get_keys_from_data_object_range
            self.ksem_event_manager.send_event_message(i_before_event_id,
                [obj_start_data, obj_end_data, i_low_comparison_operator,
                 i_high_comparison_operator, b_return_packed_data])

            list_keys = []
            b_searching = True

            cursor = self.get_cursor(kstx_transaction)
            record = cursor.first()

            # Get comparison operations
            fn_low_comparison = \
                self.get_comparison_function(i_low_comparison_operator)
            fn_high_comparison = \
                self.get_comparison_function(i_high_comparison_operator)

            if (b_return_packed_data):
                optional_data_unpacking = \
                    lambda item_packed_key, u_key_type: item_packed_key
            else:
                optional_data_unpacking = lambda item_packed_key, u_key_type: \
                    self.unmarshal_key(item_packed_key, self.u_key_type)

            while (record):
                while (b_searching):
                    current_unpickled_data = \
                        self.unpickle_data(record[self.__VALUE])
                    if (fn_low_comparison(
                        obj_start_data, current_unpickled_data) and
                        fn_high_comparison(
                        current_unpickled_data, obj_end_data)):
                            list_keys.append(optional_data_unpacking(
                                record[self.__KEY], self.u_key_type))
                            b_searching = False
                    else:
                        record = cursor.next_dup()
                        if (not record):
                            b_searching = False
                        # end if
                    # end if
                # end inner loop
                record = cursor.next()
                b_searching = True
            # end outer loop

            cursor.close()
            kstx_transaction.commit(0, self.get_keys_from_data_object_range)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_start_data, obj_end_data,
                    i_low_comparison_operator, i_high_comparison_operator,
                    list_keys])
            return list_keys

        except db.DBNotFoundError:
            cursor.close()
            kstx_transaction.cancel()
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_start_data, obj_end_data,
                    i_low_comparison_operator, i_high_comparison_operator, []])

        except KSCursorError:
            # Database is empty, so no keys are found.
            kstx_transaction.commit(0, self.get_keys_from_data_object_range)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_start_data, obj_end_data,
                    i_low_comparison_operator, i_high_comparison_operator,
                    list_keys])
            return list_keys


    def get_data_objects_from_key_range(self, obj_start_key, obj_end_key,
            i_low_comparison_operator, i_high_comparison_operator,
            kstx_parent_transaction, b_return_packed_data):
        """
        Go through the keys and find the list of data objects from the given
        values in the range of keys. It is assumed that the caller will use
        a type that would support a range of data. Range is
        obj_start_key <= key <= obj_end_key.

        """
        try:
            # Since NULL_VALUE keys are not directly searched for in the
            # database, allow NULLs to be searched. No values will be returned
            # instead of a NULL_VALUE.

            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.get_data_objects_from_key_range)
            i_before_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_BEF_get_data_objects_from_key_range
            i_after_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_AFT_get_data_objects_from_key_range
            self.ksem_event_manager.send_event_message(i_before_event_id,
                [obj_start_key, obj_end_key, i_low_comparison_operator,
                 i_high_comparison_operator, b_return_packed_data])

            list_data_objects = []

            # Rare case but possible under some odd circumstance
            if (obj_start_key == KSColumn.NULL_VALUE and
                obj_end_key == KSColumn.NULL_VALUE):
                list_data_objects.append(KSColumn.NULL_VALUE)

            cursor = self.get_cursor(kstx_transaction)
            record = cursor.first()
            b_searching = True

            # Get comparison operations
            fn_low_comparison = \
                self.get_comparison_function(i_low_comparison_operator)
            fn_high_comparison = \
                self.get_comparison_function(i_high_comparison_operator)

            if (b_return_packed_data):
                optional_data_unpickling = \
                    lambda item_pickled_data: item_pickled_data
            else:
                optional_data_unpickling = lambda item_pickled_data: \
                    self.unpickle_data(item_pickled_data)

            while (record):
                while (b_searching):
                    current_unpacked_key = self.unmarshal_key(
                        record[self.__KEY], self.u_key_type)

                    if (fn_low_comparison(
                        obj_start_key, current_unpacked_key) and
                        fn_high_comparison(
                        current_unpacked_key, obj_end_key)):
                            list_data_objects.append(optional_data_unpickling(
                                record[self.__VALUE]))
                            b_searching = False
                    else:
                        record = cursor.next_dup()
                        if (not record):
                            b_searching = False
                        # end if
                    # end if
                # end inner loop
                record = cursor.next()
                b_searching = True
            # end outer loop

            cursor.close()
            kstx_transaction.commit(0, self.get_data_objects_from_key_range)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_start_key, obj_end_key,
                    i_low_comparison_operator, i_high_comparison_operator,
                    list_data_objects])
            return list_data_objects

        except db.DBNotFoundError:
            cursor.close()
            kstx_transaction.cancel()
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_start_key, obj_end_key,
                    i_low_comparison_operator, i_high_comparison_operator, []])

        except KSCursorError:
            # No keys are returned because the database is empty.
            kstx_transaction.commit(0, self.get_data_objects_from_key_range)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_start_key, obj_end_key,
                    i_low_comparison_operator, i_high_comparison_operator,
                    list_data_objects])
            return list_data_objects


    def has_value(self, obj_value, kstx_parent_transaction):
        """
        Return a boolean value that indicates whether or not the value exists
        at least once in the database.

        """
        # Code adapted from _db_print in dbtables.py in pybsddb
        # OK to search for NULL_VALUE.
        try:
            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.has_value)
            i_before_event_id = KSColumnBSDDBDuplicate.__KSEM_BEF_has_value
            i_after_event_id = KSColumnBSDDBDuplicate.__KSEM_AFT_has_value
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [obj_value])

            cursor = self.get_cursor(kstx_transaction)
            b_found = False
            b_searching = True

            record = cursor.first()
            pickled_obj_value = self.pickle_data(obj_value)

            while (b_searching):
                if (record):
                    if (pickled_obj_value == record[self.__VALUE]):
                        b_searching = False
                        b_found = True
                    else:
                        record = cursor.next()
                    # end if
                else:
                    b_searching = False
                # end if
            # end loop

            cursor.close()
            kstx_transaction.commit(0, self.has_value)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_value, b_found])
            return b_found

        except db.DBNotFoundError:
            cursor.close()
            kstx_transaction.cancel()
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_value, b_found])
            return b_found

        except KSCursorError:
            # Technically, NULL_VALUE exists in every database; but with an
            # empty database, it is just empty with no values...not even
            # NULL. Yeah, this is splitting hairs, but this is a physical
            # storage question, not a set theory question.
            kstx_transaction.commit(0, self.has_value)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_value, False])
            return False


    def insert_item_with_key_value_pair(self, obj_key, obj_data,
                                            kstx_parent_transaction):
        """
        Insert the key/data pair. Need to watch for the case where there only
        was a NULL value assigned to this key. For this reason, the new
        value(s) should displace the NULL value.

        """
        try:
            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.insert_item_with_key_value_pair)
            i_before_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_BEF_insert_item_with_key_value_pair
            i_after_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_AFT_insert_item_with_key_value_pair
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [obj_key, obj_data])

            # OK to have NULL_VALUE values but prohibit the insertion of
            # NULL keys.
            if (obj_key == KSColumn.NULL_VALUE):
                KSColumnError.print_and_raise_error(u'The insertion of a NULL '
                    + u'key is prohibited.', KSColumnError.B_CLASS_EXCEPTION,
                    self.ksem_event_manager, i_after_event_id)

            self.__remove_null_value(obj_key, kstx_transaction)

            # Action
            packed_key = self.marshal_key(obj_key)
            packed_data = self.pickle_data(obj_data)

            cursor = self.get_cursor(kstx_transaction)
            cursor.put(packed_key, packed_data, db.DB_KEYFIRST)
            cursor.close()
            kstx_transaction.commit(0, self.insert_item_with_key_value_pair)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_key, obj_data, True])
            return True

        except KSCursorError:
            kstx_transaction.commit(0, self.insert_item_with_key_value_pair)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_key, obj_data, False])
            return False

        except Exception:
            cursor.close()
            kstx_transaction.cancel()
            KSColumnError.print_and_raise_error(u'The key/value pair could ' \
                + u'not be inserted.', KSColumnError.B_CLASS_EXCEPTION,
                self.ksem_event_manager, i_after_event_id)


    def insert_value_and_generate_key(self, obj_data, kstx_parent_transaction):
        """
        Insert data with time_id as a key. Key will be stored in big endian
        notation.

        """
        try:
            # NULL values are allowed, and the possibility of submitting a
            # NULL key is not possible for the user.

            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.insert_value_and_generate_key)
            i_before_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_BEF_insert_value_and_generate_key
            i_after_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_AFT_insert_value_and_generate_key
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [obj_data])

            obj_key = self.generate_id()
            self.insert_item_with_key_value_pair(obj_key, obj_data,
                                                 kstx_transaction)
            kstx_transaction.commit(0, self.insert_value_and_generate_key)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_key, obj_data, True])
            return True

        except Exception:
            kstx_transaction.cancel()
            KSColumnError.print_and_raise_error(u'The value and generated ' \
                + u'key could not be inserted.',
                KSColumnError.B_UNKNOWN_EXCEPTION,
                self.ksem_event_manager, i_after_event_id)


    def insert_items_with_key_and_value_list(self, obj_key, list_values,
                                                 kstx_parent_transaction):
        """
        Insert the key/values pair. Need to watch for the case where there only
        was a NULL value assigned to this key. For this reason, the new
        value(s) should displace the NULL value.

        """
        try:
            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.insert_items_with_key_and_value_list)
            i_before_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_BEF_insert_items_with_key_and_value_list
            i_after_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_AFT_insert_items_with_key_and_value_list
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [obj_key, list_values])

            if (obj_key == KSColumn.NULL_VALUE):
                KSColumnError.print_and_raise_error(u'The insertion of a NULL '
                    + u'key is prohibited.', KSColumnError.B_CLASS_EXCEPTION,
                    self.ksem_event_manager, i_after_event_id)

            self.__remove_null_value(obj_key, kstx_transaction)

            packed_key = self.marshal_key(obj_key)
            cursor = self.get_cursor(kstx_transaction)

            # Action
            for item in list_values:
                pickled_item = self.pickle_data(item)
                cursor.put(packed_key, pickled_item, db.DB_KEYFIRST)
            # end loop

            cursor.close()
            kstx_transaction.commit(0, self.insert_items_with_key_and_value_list)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_key, list_values, True])
            return True

        except KSCursorError:
            kstx_transaction.commit(0, self.insert_items_with_key_and_value_list)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_key, list_values, False])
            return False

        except Exception:
            cursor.close()
            kstx_transaction.cancel()
            KSColumnError.print_and_raise_error(u'An item from the list ' \
                + u'could not be inserted.', KSColumnError.B_UNKNOWN_EXCEPTION,
                self.ksem_event_manager, i_after_event_id)


    def insert_value_list_and_generate_key(self, list_values,
                                               kstx_parent_transaction):
        """
        Insert data with time_id as a key. Key will be stored in big endian
        notation.

        """
        try:
            # NULL values are allowed, and the possibility of submitting a
            # NULL key is not possible for the user.

            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.insert_value_list_and_generate_key)
            i_before_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_BEF_insert_value_list_and_generate_key
            i_after_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_AFT_insert_value_list_and_generate_key
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [list_values])

            obj_key = self.generate_id()
            self.insert_items_with_key_and_value_list(obj_key, list_values,
                                                        kstx_transaction)
            kstx_transaction.commit(0, self.insert_value_list_and_generate_key)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_key, list_values, True])
            return True

        except Exception:
            kstx_transaction.cancel()
            KSColumnError.print_and_raise_error(u'The value list and '
                + u'generated key could not be inserted.',
                KSColumnError.B_UNKNOWN_EXCEPTION,
                self.ksem_event_manager, i_after_event_id)


    def change_key(self, obj_old_key, obj_new_key, kstx_parent_transaction):
        """
        Get the data list from the old key, and store it as a new key
        value list.

        """
        try:
            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.change_key)
            i_before_event_id = KSColumnBSDDBDuplicate.__KSEM_BEF_change_key
            i_after_event_id = KSColumnBSDDBDuplicate.__KSEM_AFT_change_key
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [obj_old_key, obj_new_key])

            if (obj_old_key == KSColumn.NULL_VALUE or
                obj_new_key == KSColumn.NULL_VALUE):
                KSColumnError.print_and_raise_error(u'An existing key cannot '
                    + u'be changed to a NULL value.',
                    KSColumnError.B_CLASS_EXCEPTION,
                    self.ksem_event_manager, i_after_event_id)

            # Key will be packed in their functions
            list_data = self.get_data_from_key(obj_old_key,
                kstx_transaction, KSColumn.B_REGULAR_DATA)
            self.delete_item_by_key(obj_old_key, kstx_transaction)
            self.insert_items_with_key_and_value_list(obj_new_key, list_data,
                                                        kstx_transaction)
            kstx_transaction.commit(0, self.change_key)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_old_key, obj_new_key, True])
            return True

        except Exception:
            kstx_transaction.cancel()
            KSColumnError.print_and_raise_error(u'The key could not '
                + u'be changed.', KSColumnError.B_UNKNOWN_EXCEPTION,
                self.ksem_event_manager, i_after_event_id)


    # change_all_data_values in the parent class performs the same operation
    def change_key_value_pair(self, obj_key, obj_old_data, obj_new_data,
                                  kstx_parent_transaction):
        """
        Simple pass-through for __change_key_value_pair. See the private
        function for more information.

        """
        return self.__change_key_value_pair(obj_key, obj_old_data, obj_new_data,
                                  kstx_parent_transaction)


    def delete_key_value_pair(self, obj_key, obj_old_data,
                                  kstx_parent_transaction):
        """
        Alternate version of __change_key_value_pair. Simply delete the value
        only found in the list for a given key. To accomplish this, simply
        assign None to obj_new_data.

        """
        return self.__change_key_value_pair(obj_key, obj_old_data, None,
                                  kstx_parent_transaction)


    def __change_key_value_pair(self, obj_key, obj_old_data, obj_new_data,
                                  kstx_parent_transaction):
        """
        Replace the old value of a key with a new value. For duplicates, the
        old value MUST be present in order to locate it (especially in a list
        of duplicates) and add the new value.

        When None is provided for obj_new_data, just remove it from the list.

        """
        try:
            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.change_key_value_pair)
            i_before_event_id = KSColumnBSDDBDuplicate.__KSEM_BEF_change_key_value_pair
            i_after_event_id = KSColumnBSDDBDuplicate.__KSEM_AFT_change_key_value_pair
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [obj_key, obj_old_data, obj_new_data])

            if (obj_key == KSColumn.NULL_VALUE):
                KSColumnError.print_and_raise_error(u'An existing key cannot '
                    + u'be a NULL value.',
                    KSColumnError.B_CLASS_EXCEPTION,
                    self.ksem_event_manager, i_after_event_id)

            list_values = \
                self.get_data_from_key(obj_key, KSColumn.B_REGULAR_DATA)
            list_values.remove(obj_old_data)

            if (obj_new_data):
                # Only add a new value if one exists; otherwise, just delete it
                list_values.append(obj_new_data)

            self.delete_item_by_key(obj_key, kstx_transaction)
            self.insert_items_with_key_and_value_list(obj_key, list_values,
                                                        kstx_transaction)
            kstx_transaction.commit(0, self.change_key_value_pair)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [obj_key, obj_old_data, obj_new_data, True])
            return True

        except ValueError:
            kstx_transaction.cancel()
            KSColumnError.print_and_raise_error(u'The old value could not '
                + u'be found in the value list.',
                KSColumnError.B_CLASS_EXCEPTION, self.ksem_event_manager,
                i_after_event_id)

        except Exception:
            kstx_transaction.cancel()
            KSColumnError.print_and_raise_error(u'The value could not be '
                + u'changed for the given key.',
                KSColumnError.B_UNKNOWN_EXCEPTION,
                self.ksem_event_manager, i_after_event_id)


    def delete_items_by_value_list(self, list_values, kstx_parent_transaction):
        """
        Delete items having the data values in the list. Slow way: scan the
        database and swap out the old values with the new values.

        """
        # Code adapted from _db_print in dbtables.py in pybsddb
        # Ok to delete items if they are NULL.
        try:
            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.delete_items_by_value_list)
            i_before_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_BEF_delete_items_by_value_list
            i_after_event_id = \
                KSColumnBSDDBDuplicate.__KSEM_AFT_delete_items_by_value_list
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [list_values])

            # Pickle the items in the list
            list_pickled_values = []
            for value in list_values:
                list_pickled_values.append(self.pickle_data(value))

            b_searching = True
            cursor = self.get_cursor(kstx_transaction)
            record = cursor.first()

            list_keys_deleted = []
            while (record):
                while (b_searching):
                    current_pickled_data = record[self.__VALUE]

                    if (current_pickled_data in list_pickled_values):
                        list_keys_deleted.append(record[self.__KEY])
                        cursor.delete()
                        b_searching = False
                    else:
                        record = cursor.next_dup()
                        if (not record):
                            b_searching = False
                        # end if
                    # end if
                # end inner loop
                record = cursor.next()
                b_searching = True
            # end outer loop

            cursor.close()
            kstx_transaction.commit(0, self.delete_items_by_value_list)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [list_values, list_keys_deleted, True])
            return True

        except KSCursorError:
            kstx_transaction.commit(0, self.delete_items_by_value_list)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [list_values, list_keys_deleted, False])
            return False

        except db.DBNotFoundError:
            cursor.close()
            kstx_transaction.cancel()
            KSColumnError.print_and_raise_error(u'The values in the list '
                + u'could not be deleted.',
                KSColumnError.B_UNKNOWN_EXCEPTION,
                self.ksem_event_manager, i_after_event_id)


    # *********** CHANGE FOR MULTIPLE VALUES STORED FOR A SINGLE KEY
    def reorganize_keys(self, kstx_parent_transaction):
        """
        Think of this like defragmenting a hard drive except that this function
        will consolidate gaps in IDs and free up "lost" space that was created
        with the passage of time. The reorganization is dependent upon the
        structure of column data store and requires that iteration through
        the list of keys is done in ASCENDING order [lowest to highest].
        Although it seems obvious, this function is only suitable for auto-
        generated integer or long IDs.

        All data must be throughly backed up using transactions. Because all
        changes are a part of a single transaction block, code from the
        function change_key_value_pair will be duplicated and send events
        will occur for every data change.

        """
        try:
            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.reorganize_keys)
            i_before_event_id = self.__KSEM_BEF_reorganize_keys
            i_after_event_id = self.__KSEM_AFT_reorganize_keys
            self.ksem_event_manager.send_event_message(i_before_event_id, [])

            cursor = self.get_cursor(kstx_parent_transaction)
            record = cursor.first()
            i_new_id = -sys.maxint

            while record:
                list_values = []
                item_packed_key, item_pickled_data = record
                obj_old_key = self.unmarshal_key(
                    item_packed_key, self.u_key_type)
                record_inner = record

                if (i_new_id != obj_old_key):
                    # Get the data from the list in a key
                    while record_inner:
                        list_values.append(
                            self.unpickle_data(record[self.__VALUE]))
                        record_inner.cursor.next_dup()
                    # end loop

                    # Action: Delete the old key and insert the value list
                    self.__db_column.delete(obj_old_key,
                        txn=kstx_transaction.obj_transaction)
                    #del self.__db_column[obj_old_key]

                    for item in list_values:
                        pickled_item = self.pickle_data(item)
                        cursor.put(
                            item_packed_key, pickled_item, db.DB_KEYFIRST)
                    # end loop

                    # Send out event notification to listening columns
                    self.ksem_event_manager.send_event_message(
                        i_after_event_id, [obj_old_key, i_new_id])
                # else, nothing

                # Move cursor and increment ID counter
                i_new_id = i_new_id + 1
                item = cursor.next_nodup()

            # end loop

            # Now, recalculate the new expiration time for the column
            i_remaining_id_count = \
                int((sys.maxint - self.size()) / KSUtility.MAX_IDS_PER_SECOND)
            i_available_id_count = \
                i_remaining_id_count - KSColumn.MIN_REMAINING_IDS

            mx_new_start_datetime = self.mx_start_datetime + \
                mx.DateTime.DateTimeDeltaFromSeconds(i_remaining_id_count)
            mx_new_last_datetime = mx_new_start_datetime + \
                mx.DateTime.DateTimeDeltaFromSeconds(i_available_id_count)

            self.change_datetimes(mx_new_start_datetime, mx_new_last_datetime)
            cursor.close()
            kstx_transaction.commit(0, self.reorganize_keys)

        except KSCursorError:
            kstx_transaction.commit(0, self.reorganize_keys)

        except Exception:
            cursor.close()
            kstx_transaction.cancel()


    def print_all(self, kstx_parent_transaction):
        try:
            kstx_transaction = self.get_transaction(kstx_parent_transaction,
                self.print_all)
            i_before_event_id = KSColumnBSDDBDuplicate.__KSEM_BEF_print_all
            i_after_event_id = KSColumnBSDDBDuplicate.__KSEM_AFT_print_all
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [])

            cursor = self.get_cursor(kstx_transaction)
            record = cursor.first()
            duplicate_record = 1

            while (record):
                packed_key, obj_pickled_data = record
                sys.stdout.write('%s: ' % (self.unmarshal_key(
                    packed_key, self.u_key_type)))
                sys.stdout.write('%s' % (self.unpickle_data(
                    obj_pickled_data)))

                duplicate_record = cursor.next_dup()
                while (duplicate_record):
                    sys.stdout.write(', %s' %
                        (self.unpickle_data(duplicate_record[1])))
                    duplicate_record = cursor.next_dup()
                # end loop

                sys.stdout.write('\n')
                record = cursor.next()
            # end loop

            cursor.close()
            kstx_transaction.commit(0, self.print_all)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [True])
            return True

        except KSCursorError:
            kstx_transaction.commit(0, self.print_all)
            self.ksem_event_manager.send_event_message(
                i_after_event_id, [False])
            return False

        except db.DBNotFoundError:
            cursor.close()
            kstx_transaction.cancel()
            self.ksem_event_manager.send_event_message(
                i_before_event_id, [[]])


    def __custom_unpickle_data(self, obj_key, u_type):
        # Ignore the u_type in this case in order to keep code from getting
        # to tangled.
        return self.unpickle_data(obj_key)


    def __remove_null_value(self, obj_key, kstx_parent_transaction):
        """
        Remove an existing NULL value if a single item or multiple items
        seeks to replace this "placeholder". No need to check for NULL keys
        because this check is made before this function is called in the
        referring functions.

        """
        kstx_transaction = self.get_transaction(kstx_parent_transaction,
            self.__remove_null_value)
        if (self.has_key(obj_key, kstx_transaction)):
            list_values = self.get_data_from_key(obj_key, kstx_transaction,
                KSColumn.B_REGULAR_DATA)

            # The list should only have one NULL value; if not, make sure
            # to remove only one NULL value and return the remaining values.
            if (KSColumn.NULL_VALUE in list_values):
                i_number_of_items = len(list_values)
                if (i_number_of_items > 1):
                    i_null_location = list_values.index(KSColumn.NULL_VALUE)
                    del list_values[i_null_location]
                    self.insert_items_with_key_and_value_list(
                        obj_key, list_values, kstx_transaction)
                elif (i_number_of_items == 1):
                    self.delete_item_by_key(obj_key, kstx_transaction)
                # else, ignore